import math
import jieba
import jieba.posseg as psg
def load_data(pos=False,corpus_path='corpus.txt'):
    doc_list=[]
    for line in open(corpus_path,'r',encoding='utf-8'):
        content=line.strip()
        cut_list=cutWord(content,pos)
        filter_list=removeWord(cut_list,pos)
        doc_list.append(filter_list)
    return doc_list
def cutWord(sentence,pos=False):
    if not pos:
        cut_list=jieba.cut(sentence)
    else:
        cut_list=psg.cut(sentence)
    return cut_list
def removeWord(seg_list,pos=False):
    stop_word_path='stops_list.txt'
    stopword_list=[sw.replace('\n','')for sw in open(stop_word_path,'r',encoding='utf-8').readlines()]
    filter_list=[]
    for seg in seg_list:
        if not pos:
            word=seg
            flag='n'
        else:
            word=seg.word
            flag=seg.flag
        if not flag.startswith('n'):
            continue
        if not word in stopword_list and len(word)>1:
            filter_list.append(word)
    return filter_list
def get_tf(word_list):
    tf_dic={}
    for word in word_list:
        tf_dic[word]=tf_dic.get(word,0.0)+1.0
    tt_count=len(word_list)
    for k,v in tf_dic.items():
        tf_dic[k]=float(v)/tt_count
    return tf_dic
def get_idf(doc_list):
    idf_dic={}
    tt_count=len(doc_list)
    for doc in doc_list:
        for word in set(doc):
            idf_dic[word]=idf_dic.get(word,0.0)+1.0
    for k,v in idf_dic.items():
        idf_dic[k]=math.log(tt_count/(1.0+v))
    default_idf=math.log(tt_count/(1.0))
    return idf_dic,default_idf
def get_tfidf(idf_dic,default_idf,word_list,keyword_num):
    tfidf_dic={}
    for word in word_list:
        idf=idf_dic.get(word,default_idf)
        tf_dic=get_tf(word_list)
        tf=tf_dic.get(word,0)
        tfidf=tf*idf
        tfidf_dic[word]=tfidf
    for k,v in sorted(tfidf_dic.items(),key=lambda x:x[1],reverse=True)[:keyword_num]:
        print(k+"",end='')
def tfidf_extract(word_list,pos=False,keyword_num=10):
    doc_list=load_data(pos)
    idf_dic,default_idf=get_idf(doc_list)
    tfidf_model=get_tfidf(idf_dic,default_idf,word_list,keyword_num)
if __name__=='__main__':
    text='冬奥会会徽以汉字“冬”为灵感来源，运用中国书法的艺术形态，将厚重的东方文化底蕴与国际化的现代风格融为一体，呈现出新时代的中国新形象、新梦想，传递出新时代中国为办好北京冬奥会，圆冬奥之梦，实现“三亿人参与冰雪运动”目标，圆体育强国之梦，推动世界冰雪运动发展，为国际奥林匹克运动做出新贡献的不懈努力和美好追求。会徽图形上半部分展现滑冰运动员的造型，下半部分表现滑雪运动员的英姿。中间舞动的线条流畅且充满韵律，代表举办地起伏的山峦、赛场、冰雪滑道和节日飘舞的丝带，为会徽增添了节日喜庆的视觉感受，也象征着北京冬奥会将在中国春节期间举行。会徽以蓝色为主色调，寓意梦想与未来，以及冰雪的明亮纯洁。红黄两色源自中国国旗，代表运动的激情、青春与活力。在“BEIJING 2022”字体的形态上汲取了中国书法与剪纸的特点，增强了字体的文化内涵和表现力，也体现了与会徽图形的整体感和统一性。'
    cut_list=cutWord(text,pos=True)
    filter_list=removeWord(cut_list,pos=True)
    print('TF-IDF算法关键词提取的结果:')
    tfidf_extract(filter_list)

